import requests
from lxml import etree
from redisDemo import getInstanct
import mysqlDemo as mysql

rd = getInstanct()

BASE_PATH = "https://www.kuaiyankanshu.net"
SQL = "insert into xbtx(url,title,content) values('%s','%s','%s')"

flag = 0
#with open("xbtx.txt","a",encoding="utf-8") as f:
while True:
    #获取地址
    if flag == 0:
        item = rd.rpop("xsList")

    if item is None:
        break

    if isinstance(item,bytes):
        item = bytes.decode(item)

    url = BASE_PATH + item
    print(url)

    #请求地址
    response = requests.get(url)
    response.encoding = "utf-8"
    root = etree.HTML(response.content)

    #解析
    title = root.xpath("//div[@class='title']/h1/a/text()")[0]
    print(title)
    content = root.xpath("//div[@class='content']/text()")
    print(len(content))
    if len(content) < 10 and flag < 5:
        item = root.xpath("//ul[@class='dropmenu-item']/li/a/@href")
        if len(item) <= flag:
            flag = 0
            continue
        
        item = item[flag]
        print("redirect:%s" % item)
        flag += 1
        continue
    else:
        flag = 0

    content = "\n".join(content)
    #print(content)

    #生成到文件
    # f.write(title+"\n")
    # f.write(content + "\n\n")

    #写库
    data = (url,title,content)
    mysql.exec(SQL % data)

mysql.close()
